# -*- coding: utf-8 -*-
import re,os,sqlite3,urllib2,datetime,zlib,gzip
from pyquery import PyQuery as pq

def htmlclean(html):
    # 多于1个的换行、tab和&nbsp;，替换为一个空格
    pattern='\s+|(&nbsp;)+'
    html=re.sub(pattern,' ',html)
    # 去掉html标记间的空格
    # pattern='> <'
    # html=re.sub(pattern,'><',html)
    html=html.replace('> <','><')
    # 去掉object标签和多余的商品行
    pattern='<tr><td valign="top"><table width="99%" height="44" border="0" align="right" cellpadding="0" cellspacing="0"><tr><td></td></tr></table></td></tr>|<object.+?</object>'
    html=re.sub(pattern,'',html)
    # 通过pyquery来规格化html，同时去掉了meta标签
    return pq(html).outerHtml()


def listdir2(arg,dir,files):
    for f in files:
        print os.path.join(dir,f)
# os.path.walk('3104',listdir2,())
    
fname='310120110517964514'
html=open(fname).read()
html=htmlclean(html)
d=pq(html)
f=open(fname+'.d','w')
f.write(d.outerHtml().encode('gbk'))
f.close()

d=pq(open(fname+'.d').read().decode('gbk'))
# pattern=r'<tr><td valign="top"><table width="99%" height="44" border="0" align="right" cellpadding="0" cellspacing="0">.+?</table></td></tr>'
# s=re.search(pattern,html).start()
# r=re.findall(pattern,html)
# j=''.join(r[:5])
# print j,len(j),s
# print html[len(j)+s:]

# html=open('310120110517964514.t').read()

# pattern=r'<tr><td valign="top"><table width="99%" height="44" border="0" align="right" cellpadding="0" cellspacing="0"><tr><td></td></tr></table></td></tr>'
# print re.search(pattern,html).start()
# print re.search(pattern,html).end()
# print re.search(pattern,html).span()
# print html[re.search(pattern,html).start():re.search(pattern,html).end()]


# A类企业
print d('#Div1').text()
# 310120110517964514
print d('div[style="position:absolute; width:248px; height:19px; z-index:3; left: 512px; top: 94px;"]').text()
# 北仑海关
print d('div[style="position:absolute; width:130px; height:16px; z-index:2; left: 66px; top: 132px;"]').text()
# (3104)
print d('div[style="position:absolute; width:107px; height:16px; z-index:2; left: 204px; top: 132px;"]').text()
# 20120124
print d('div[style="position:absolute; width:115px; height:18px; z-index:2; left: 452px; top: 132px;"]').text()
# 2011-12-31
print d('div[style="position:absolute; width:126px; height:18px; z-index:2; left: 605px; top: 132px;"]').text()
# 宁波得力进出口有限公司
print d('div[style="position:absolute; width:232px; height:18px; z-index:2; left: 80px; top: 154px;"]').text()
# 3302964471
print d('div[style="position:absolute; width:114px; height:15px; z-index:2; left: 36px; top: 165px;"]').text()
# 33029C4102
print d('div[style="position:absolute; width:114px; height:15px; z-index:2; left: 35px; top: 196px;"]').text()
# 宁波得力文具有限公司
print d('div[style="position:absolute; width:232px; height:18px; z-index:2; left: 80px; top: 183px;"]').text()
# 水路运输
print d('div[style="position:absolute; width:67px; height:15px; z-index:2; left: 313px; top: 163px;"]').text()
# KAIPING/W1178
print d('div[style="position:absolute; width:191px; height:15px; z-index:2; left: 380px; top: 164px;"]').text()
# 177CBJBJN06067A
print d('div[style="position:absolute; width:160px; height:15px; z-index:2; left: 571px; top: 165px;"]').text()
# 一般贸易   (0110)
print d('div[style="position:absolute; width:142px; height:15px; z-index:2; left: 331px; top: 196px;"]>div').text()
# 一般征税   (101)
print d('div[style="position:absolute; width:140px; height:15px; z-index:2; left: 462px; top: 193px;"]>div').text()
# 电汇(T/T)
print d('div[style="position:absolute; width:119px; height:15px; z-index:2; left: 613px; top: 193px;"]').text()
# 塞拉利昂   (242)
print d('div[style="position:absolute; width:180px; height:15px; z-index:2; left: 216px; top: 221px;"]').text()

print d('div[style="position:absolute; width:184px; height:15px; z-index:2; left: 400px; top: 223px;"]').text()
print d('div[style="position:absolute; width:139px; height:15px; z-index:2; left: 586px; top: 223px;"]').text()
print d('div[style="position:absolute; width:143px; height:15px; z-index:2; left: 68px; top: 249px;"]').text()
print d('div[style="position:absolute; width:65px; height:15px; z-index:2; left: 233px; top: 251px;"]').text()
print d('div[style="position:absolute; width:142px; height:15px; z-index:2; left: 309px; top: 252px;"]').text()
print d('div[style="position:absolute; width:118px; height:15px; z-index:2; left: 463px; top: 253px;"]').text()
print d('div[style="position:absolute; width:142px; height:15px; z-index:2; left: 590px; top: 251px;"]').text()
print d('div[style="position:absolute; width:143px; height:15px; z-index:2; left: 77px; top: 282px;"]').text()
print d('div[style="position:absolute; width:103px; height:15px; z-index:2; left: 228px; top: 286px;"]').text()
print d('div[style="position:absolute; width:103px; height:15px; z-index:2; left: 358px; top: 282px;"]').text()
print d('div[style="position:absolute; width:103px; height:15px; z-index:2; left: 479px; top: 285px;"]').text()
print d('div[style="position:absolute; width:103px; height:15px; z-index:2; left: 627px; top: 285px;"]').text()
print d('div[style="position:absolute; width:175px; height:15px; z-index:2; left: 66px; top: 320px;"]').text()
print d('div[style="position:absolute; width:714px; height:62px; z-index:6; left: 24px; top: 349px;"]').text()
print d('div[style="position:absolute; width:286px; height:15px; z-index:2; left: 260px; top: 316px;"]').text()

# table=d('table[height="233"]')
# table.append(table.html())
open('t.t','w').write(d.outerHtml().encode('gbk'))


cds=[]
def listdir(root,level=1):
    for dirs in os.listdir(root):
        path=os.path.join(root,dirs)
        if level==4:
            cds.append(path)
        if os.path.isdir(path):
            listdir(path,level+1)
listdir('3104')
print cds
def getcdfilerange(dir):
    cd=dir.replace(os.sep,'').split('-')
    s=int(cd[0])
    e=int(cd[0][:11]+cd[1])
    if e>s:
        return s,e+1
    else:
        return e,s+1
def getcdpath(dir):
    # return dir.split('-')[0][:-7]
    return (os.sep).join(dir.split(os.sep)[:4])
for c in cds:
    cdpath=getcdpath(c)
    cdfilerange=getcdfilerange(c)
    print cdpath,cdfilerange
    cdrange=cdfilerange[1]-cdfilerange[0]+1
    for cdfile in range(cdfilerange[0],cdfilerange[1]):
        cd=cdpath+os.sep+str(cdfile)
        # print cd
        if os.path.isfile(cd):
            # print cd
            html=open(cd).read()
            source=htmlclean(html)
        